{
    "cells": [
        {
            "cell_type": "code",
            "execution_count": 17,
            "id": "9080b39e",
            "metadata": {},
            "outputs": [],
            "source": [
                "import logging, sys\n",
                "logging.basicConfig(stream=sys.stdout, level=logging.INFO)\n",
                "logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))\n",
                "\n",
                "# Uncomment if you want to temporarily disable logger\n",
                "logging.disable(sys.maxsize)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 2,
            "id": "7de92ce3",
            "metadata": {},
            "outputs": [],
            "source": [
                "# NOTE: only necessary for querying with `use_async=True` in notebook\n",
                "import nest_asyncio\n",
                "nest_asyncio.apply()"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 3,
            "id": "f1a9eb90-335c-4214-8bb6-fd1edbe3ccbd",
            "metadata": {},
            "outputs": [],
            "source": [
                "# My OpenAI Key\n",
                "import os\n",
                "os.environ['OPENAI_API_KEY'] = \"\""
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 3,
            "id": "8d0b2364-4806-4656-81e7-3f6e4b910b5b",
            "metadata": {},
            "outputs": [],
            "source": [
                "from gpt_index import GPTTreeIndex, SimpleDirectoryReader, LLMPredictor, GPTSimpleVectorIndex, GPTListIndex, Prompt, ServiceContext\n",
                "from gpt_index.indices.base import BaseGPTIndex\n",
                "from gpt_index.langchain_helpers.text_splitter import TokenTextSplitter\n",
                "from langchain.chat_models import ChatOpenAI\n",
                "from langchain.llms import OpenAI\n",
                "from gpt_index.response.schema import Response\n",
                "import pandas as pd\n",
                "from typing import Tuple"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "707662e5",
            "metadata": {},
            "source": [
                "# Setup data"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 2,
            "id": "b4b4387b-413e-4016-ba1e-88b3d9410a38",
            "metadata": {},
            "outputs": [],
            "source": [
                "# fetch \"New York City\" page from Wikipedia\n",
                "from pathlib import Path\n",
                "\n",
                "import requests\n",
                "response = requests.get(\n",
                "    'https://en.wikipedia.org/w/api.php',\n",
                "    params={\n",
                "        'action': 'query',\n",
                "        'format': 'json',\n",
                "        'titles': 'New York City',\n",
                "        'prop': 'extracts',\n",
                "        # 'exintro': True,\n",
                "        'explaintext': True,\n",
                "    }\n",
                ").json()\n",
                "page = next(iter(response['query']['pages'].values()))\n",
                "nyc_text = page['extract']\n",
                "\n",
                "data_path = Path('data')\n",
                "if not data_path.exists():\n",
                "    Path.mkdir(data_path)\n",
                "\n",
                "with open('data/nyc_text.txt', 'w') as fp:\n",
                "    fp.write(nyc_text)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 4,
            "id": "523fbebe-6e79-4d7b-b400-188b711a0e8f",
            "metadata": {
                "tags": []
            },
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "DEBUG:gpt_index.readers.file.base:> [SimpleDirectoryReader] Total files added: 1\n",
                        "> [SimpleDirectoryReader] Total files added: 1\n"
                    ]
                }
            ],
            "source": [
                "documents = SimpleDirectoryReader('data').load_data()"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "f4a269bd",
            "metadata": {},
            "source": [
                "# Setup benchmark"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 5,
            "id": "62f01ddf",
            "metadata": {},
            "outputs": [],
            "source": [
                "from dataclasses import dataclass\n",
                "from typing import List"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 6,
            "id": "4ff13cd4",
            "metadata": {},
            "outputs": [],
            "source": [
                "@dataclass\n",
                "class TestCase:\n",
                "    query: str \n",
                "    must_contain: List[str]"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 7,
            "id": "9c653b72",
            "metadata": {},
            "outputs": [],
            "source": [
                "@dataclass\n",
                "class TestOutcome:\n",
                "    test: TestCase\n",
                "    response: Response\n",
                "    \n",
                "    @property\n",
                "    def is_correct_response(self) -> bool:\n",
                "        is_correct = True\n",
                "        for answer in self.test.must_contain:\n",
                "            if answer not in self.response.response:\n",
                "                is_correct = False\n",
                "        return is_correct\n",
                "    \n",
                "    @property\n",
                "    def is_correct_source(self) -> bool:\n",
                "        is_correct = True\n",
                "        for answer in self.test.must_contain:\n",
                "            if all(answer not in node.source_text for node in self.response.source_nodes):\n",
                "                is_correct = False\n",
                "        return is_correct"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 8,
            "id": "b9cd18ae",
            "metadata": {},
            "outputs": [],
            "source": [
                "class Benchmark:\n",
                "    def __init__(self, tests: List[TestCase]) -> None:\n",
                "        self._tests = tests\n",
                "    \n",
                "    def test(self, index: BaseGPTIndex, llm_predictor: LLMPredictor, **kwargs) -> List[TestOutcome]:\n",
                "        outcomes: List[TestOutcome] = []\n",
                "        service_context = ServiceContext.from_defaults(llm_predictor=llm_predictor)\n",
                "        for test in self._tests:\n",
                "            response = index.query(\n",
                "                test.query,\n",
                "                service_context=service_context,\n",
                "                **kwargs\n",
                "            )\n",
                "            outcome = TestOutcome(test=test, response=response)\n",
                "            outcomes.append(outcome)\n",
                "        return outcomes"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 9,
            "id": "8edad985",
            "metadata": {},
            "outputs": [],
            "source": [
                "def analyze_outcome(outcomes: List[TestOutcome]) -> None:\n",
                "    rows = []\n",
                "    for outcome in outcomes:\n",
                "        row = [outcome.test.query, outcome.is_correct_response, outcome.is_correct_source]\n",
                "        rows.append(row)\n",
                "    df = pd.DataFrame(rows, columns=['Test Query', 'Correct Response', 'Correct Source'])\n",
                "    return df"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 10,
            "id": "4bc38077",
            "metadata": {},
            "outputs": [],
            "source": [
                "test_battle = TestCase(\n",
                "    query=\"What battles took place in New York City in the American Revolution?\",\n",
                "    must_contain=[\"Battle of Long Island\"]\n",
                ")\n",
                "\n",
                "test_mayor = TestCase(\n",
                "    query='Who was elected as the mayor after the Great Depression?',\n",
                "    must_contain=[\"Fiorello La Guardia\"]\n",
                ")\n",
                "\n",
                "test_tourists = TestCase(\n",
                "    query='How many tourists visited New York City in 2019?',\n",
                "    must_contain=['66.6 million']\n",
                ")\n",
                "test_airport = TestCase(\n",
                "    query='What are the airports in New York City?',\n",
                "    must_contain=['LaGuardia Airport']\n",
                ")\n",
                "test_visit = TestCase(\n",
                "    query='When was the first documented visit into New York Harbor?',\n",
                "    must_contain=['1524']\n",
                ")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 11,
            "id": "f159dadb",
            "metadata": {},
            "outputs": [],
            "source": [
                "bm = Benchmark([\n",
                "    test_battle,\n",
                "    test_mayor,\n",
                "    test_tourists,\n",
                "    test_airport,\n",
                "    test_visit,\n",
                "])"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "65ddbd56",
            "metadata": {},
            "source": [
                "# LLM based evaluation"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 592,
            "id": "ed175de5",
            "metadata": {},
            "outputs": [],
            "source": [
                "from gpt_index.prompts.prompt_type import PromptType\n",
                "\n",
                "EVAL_PROMPT_TMPL = (\n",
                "    \"Given the question below. \\n\"\n",
                "    \"---------------------\\n\"\n",
                "    \"{query_str}\"\n",
                "    \"\\n---------------------\\n\"\n",
                "    \"Decide if the following retreived context is relevant. \\n\"\n",
                "    \"\\n---------------------\\n\"\n",
                "    \"{context_str}\"\n",
                "    \"\\n---------------------\\n\"\n",
                "    \"Then decide if the answer is correct. \\n\"\n",
                "    \"\\n---------------------\\n\"\n",
                "    \"{answer_str}\"\n",
                "    \"\\n---------------------\\n\"\n",
                "    \"Answer in the following format:\\n\"\n",
                "    \"'Context is relevant: <True>\\nAnswer is correct: <True>' \"\n",
                "    \"and explain why.\"\n",
                ")\n",
                "\n",
                "class EvalPrompt(Prompt):\n",
                "    prompt_type: PromptType = PromptType.CUSTOM\n",
                "    input_variables: List[str] = [\"query_str\", 'context_str', 'answer_str']\n",
                "\n",
                "DEFAULT_EVAL_PROMPT = EvalPrompt(EVAL_PROMPT_TMPL)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 593,
            "id": "93c498b6",
            "metadata": {},
            "outputs": [],
            "source": [
                "import re\n",
                "def extract_eval_result(result_str: str):\n",
                "    boolean_pattern = r\"(True|False)\"\n",
                "    matches = re.findall(boolean_pattern, result_str)\n",
                "    return [match == \"True\" for match in matches]    "
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 594,
            "id": "4c8109c3",
            "metadata": {},
            "outputs": [],
            "source": [
                "def analyze_outcome_llm_single(outcome: TestOutcome, llm_predictor: LLMPredictor) -> Tuple[bool, bool]:\n",
                "    try:\n",
                "        source_text = outcome.response.source_nodes[0].source_text\n",
                "    except:\n",
                "        source_text = \"Failed to retrieve any context\"\n",
                "    result_str, _ = llm_predictor.predict(\n",
                "        DEFAULT_EVAL_PROMPT,\n",
                "        query_str=outcome.test.query,\n",
                "        context_str=source_text,\n",
                "        answer_str=outcome.response.response\n",
                "    )\n",
                "    is_context_relevant, is_answer_correct = extract_eval_result(result_str)\n",
                "    return is_answer_correct, is_context_relevant, result_str\n",
                "\n",
                "def analyze_outcome_llm(outcomes: List[TestOutcome], llm_predictor: LLMPredictor) -> None:\n",
                "    rows = []\n",
                "    for outcome in outcomes:\n",
                "        is_correct_response, is_correct_source, result_str = analyze_outcome_llm_single(outcome, llm_predictor)\n",
                "        row = [outcome.test.query, is_correct_response, is_correct_source, result_str]\n",
                "        rows.append(row)\n",
                "    df = pd.DataFrame(rows, columns=['Test Query', 'Correct Response (LLM)', 'Correct Source (LLM)', 'Eval (LLM)'])\n",
                "    return df"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "5a9f43a6",
            "metadata": {},
            "source": [
                "# Build Indices"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 643,
            "id": "790bad05",
            "metadata": {},
            "outputs": [],
            "source": [
                "vector_index = GPTSimpleVectorIndex.from_documents(\n",
                "    documents, \n",
                ")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 473,
            "id": "64c970e0",
            "metadata": {},
            "outputs": [],
            "source": [
                "list_index = GPTListIndex.from_documents(\n",
                "    documents, \n",
                ")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 468,
            "id": "bacc4f1c",
            "metadata": {},
            "outputs": [],
            "source": [
                "tree_index = GPTTreeIndex.from_documents(documents)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 632,
            "id": "a600d4de",
            "metadata": {},
            "outputs": [],
            "source": [
                "# Save indices\n",
                "vector_index.save_to_disk('vector_index.json')\n",
                "tree_index.save_to_disk('tree_index.json')\n",
                "list_index.save_to_disk('list_index.json')"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 13,
            "id": "5eec265d-211b-4f26-b05b-5b4e7072bc6e",
            "metadata": {},
            "outputs": [],
            "source": [
                "# Load indices\n",
                "tree_index = GPTTreeIndex.load_from_disk('tree_index.json')\n",
                "list_index = GPTListIndex.load_from_disk('list_index.json')\n",
                "vector_index = GPTSimpleVectorIndex.load_from_disk('vector_index.json')"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "5b2e7fdd",
            "metadata": {},
            "source": [
                "# Create LLMPredictors"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 12,
            "id": "4766ac56-ac8d-4f33-b994-6901964241ea",
            "metadata": {
                "tags": []
            },
            "outputs": [],
            "source": [
                "# gpt-4\n",
                "llm_predictor_gpt4 = LLMPredictor(\n",
                "    llm=ChatOpenAI(temperature=0, model_name=\"gpt-4\")\n",
                ")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 169,
            "id": "c8692cf6",
            "metadata": {},
            "outputs": [],
            "source": [
                "# gpt-3 (text-davinci-003)\n",
                "llm_predictor_gpt3 = LLMPredictor(llm=OpenAI(temperature=0, model_name=\"text-davinci-003\"))"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 22,
            "id": "fb74ec62",
            "metadata": {},
            "outputs": [],
            "source": [
                "# chatgpt (gpt-3.5-turbo)\n",
                "llm_predictor_chatgpt = LLMPredictor(llm=ChatOpenAI(temperature=0, model_name=\"gpt-3.5-turbo\"))"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "1354f668",
            "metadata": {},
            "source": [
                "# Benchmarking "
            ]
        },
        {
            "cell_type": "markdown",
            "id": "01124a3f",
            "metadata": {},
            "source": [
                "### Tree Index + GPT4"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 583,
            "id": "6f418554",
            "metadata": {},
            "outputs": [],
            "source": [
                "outcomes_tree_gpt4 = bm.test(tree_index, llm_predictor_gpt4)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 584,
            "id": "de98ceba",
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<div>\n",
                            "<style scoped>\n",
                            "    .dataframe tbody tr th:only-of-type {\n",
                            "        vertical-align: middle;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe tbody tr th {\n",
                            "        vertical-align: top;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe thead th {\n",
                            "        text-align: right;\n",
                            "    }\n",
                            "</style>\n",
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th></th>\n",
                            "      <th>Test Query</th>\n",
                            "      <th>Correct Response</th>\n",
                            "      <th>Correct Source</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <th>0</th>\n",
                            "      <td>What battles took place in New York City in th...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>1</th>\n",
                            "      <td>Who was elected as the mayor after the Great D...</td>\n",
                            "      <td>False</td>\n",
                            "      <td>False</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>2</th>\n",
                            "      <td>How many tourists visited New York City in 2019?</td>\n",
                            "      <td>False</td>\n",
                            "      <td>False</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>3</th>\n",
                            "      <td>What are the airports in New York City?</td>\n",
                            "      <td>False</td>\n",
                            "      <td>False</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>4</th>\n",
                            "      <td>When was the first documented visit into New Y...</td>\n",
                            "      <td>False</td>\n",
                            "      <td>False</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>\n",
                            "</div>"
                        ],
                        "text/plain": [
                            "                                          Test Query  Correct Response  \\\n",
                            "0  What battles took place in New York City in th...              True   \n",
                            "1  Who was elected as the mayor after the Great D...             False   \n",
                            "2   How many tourists visited New York City in 2019?             False   \n",
                            "3            What are the airports in New York City?             False   \n",
                            "4  When was the first documented visit into New Y...             False   \n",
                            "\n",
                            "   Correct Source  \n",
                            "0            True  \n",
                            "1           False  \n",
                            "2           False  \n",
                            "3           False  \n",
                            "4           False  "
                        ]
                    },
                    "execution_count": 584,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "analyze_outcome(outcomes_tree_gpt4)"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "f5ef33a0",
            "metadata": {},
            "source": [
                "### Tree Index + GPT3"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 549,
            "id": "ba871d2a",
            "metadata": {},
            "outputs": [],
            "source": [
                "outcomes_tree_gpt3 = bm.test(tree_index, llm_predictor_gpt3)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 550,
            "id": "7d4c6930",
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<div>\n",
                            "<style scoped>\n",
                            "    .dataframe tbody tr th:only-of-type {\n",
                            "        vertical-align: middle;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe tbody tr th {\n",
                            "        vertical-align: top;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe thead th {\n",
                            "        text-align: right;\n",
                            "    }\n",
                            "</style>\n",
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th></th>\n",
                            "      <th>Test Query</th>\n",
                            "      <th>Correct Response</th>\n",
                            "      <th>Correct Source</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <th>0</th>\n",
                            "      <td>What battles took place in New York City in th...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>False</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>1</th>\n",
                            "      <td>Who was elected as the mayor after the Great D...</td>\n",
                            "      <td>False</td>\n",
                            "      <td>False</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>2</th>\n",
                            "      <td>How many tourists visited New York City in 2019?</td>\n",
                            "      <td>False</td>\n",
                            "      <td>False</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>3</th>\n",
                            "      <td>What are the airports in New York City?</td>\n",
                            "      <td>True</td>\n",
                            "      <td>False</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>4</th>\n",
                            "      <td>When was the first documented visit into New Y...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>False</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>\n",
                            "</div>"
                        ],
                        "text/plain": [
                            "                                          Test Query  Correct Response  \\\n",
                            "0  What battles took place in New York City in th...              True   \n",
                            "1  Who was elected as the mayor after the Great D...             False   \n",
                            "2   How many tourists visited New York City in 2019?             False   \n",
                            "3            What are the airports in New York City?              True   \n",
                            "4  When was the first documented visit into New Y...              True   \n",
                            "\n",
                            "   Correct Source  \n",
                            "0           False  \n",
                            "1           False  \n",
                            "2           False  \n",
                            "3           False  \n",
                            "4           False  "
                        ]
                    },
                    "execution_count": 550,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "analyze_outcome(outcomes_tree_gpt3)"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "30a9ba34",
            "metadata": {},
            "source": [
                "### List Index + GPT4"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 18,
            "id": "bc0f05d1",
            "metadata": {
                "scrolled": true
            },
            "outputs": [],
            "source": [
                "outcomes_list_gpt4 = bm.test(list_index, llm_predictor_gpt4, response_mode=\"tree_summarize\", use_async=True)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 19,
            "id": "2d2e879d",
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<div>\n",
                            "<style scoped>\n",
                            "    .dataframe tbody tr th:only-of-type {\n",
                            "        vertical-align: middle;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe tbody tr th {\n",
                            "        vertical-align: top;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe thead th {\n",
                            "        text-align: right;\n",
                            "    }\n",
                            "</style>\n",
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th></th>\n",
                            "      <th>Test Query</th>\n",
                            "      <th>Correct Response</th>\n",
                            "      <th>Correct Source</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <th>0</th>\n",
                            "      <td>What battles took place in New York City in th...</td>\n",
                            "      <td>False</td>\n",
                            "      <td>True</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>1</th>\n",
                            "      <td>Who was elected as the mayor after the Great D...</td>\n",
                            "      <td>False</td>\n",
                            "      <td>True</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>2</th>\n",
                            "      <td>How many tourists visited New York City in 2019?</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>3</th>\n",
                            "      <td>What are the airports in New York City?</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>4</th>\n",
                            "      <td>When was the first documented visit into New Y...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>\n",
                            "</div>"
                        ],
                        "text/plain": [
                            "                                          Test Query  Correct Response  \\\n",
                            "0  What battles took place in New York City in th...             False   \n",
                            "1  Who was elected as the mayor after the Great D...             False   \n",
                            "2   How many tourists visited New York City in 2019?              True   \n",
                            "3            What are the airports in New York City?              True   \n",
                            "4  When was the first documented visit into New Y...              True   \n",
                            "\n",
                            "   Correct Source  \n",
                            "0            True  \n",
                            "1            True  \n",
                            "2            True  \n",
                            "3            True  \n",
                            "4            True  "
                        ]
                    },
                    "execution_count": 19,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "analyze_outcome(outcomes_list_gpt4)"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "8cba793c",
            "metadata": {},
            "source": [
                "### List Index + GPT3"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 501,
            "id": "66cfa3fa",
            "metadata": {},
            "outputs": [],
            "source": [
                "outcomes_list_gpt3 = bm.test(list_index, llm_predictor_gpt3, response_mode=\"tree_summarize\", use_async=True)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 502,
            "id": "06bc98d8",
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<div>\n",
                            "<style scoped>\n",
                            "    .dataframe tbody tr th:only-of-type {\n",
                            "        vertical-align: middle;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe tbody tr th {\n",
                            "        vertical-align: top;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe thead th {\n",
                            "        text-align: right;\n",
                            "    }\n",
                            "</style>\n",
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th></th>\n",
                            "      <th>Test Query</th>\n",
                            "      <th>Correct Response</th>\n",
                            "      <th>Correct Source</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <th>0</th>\n",
                            "      <td>What battles took place in New York City in th...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>1</th>\n",
                            "      <td>Who was elected as the mayor during the Great ...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>2</th>\n",
                            "      <td>How many tourists visited New York City in 2019?</td>\n",
                            "      <td>False</td>\n",
                            "      <td>True</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>3</th>\n",
                            "      <td>What are the airports in New York City?</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>4</th>\n",
                            "      <td>When was the first documented visit into New Y...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>\n",
                            "</div>"
                        ],
                        "text/plain": [
                            "                                          Test Query  Correct Response  \\\n",
                            "0  What battles took place in New York City in th...              True   \n",
                            "1  Who was elected as the mayor during the Great ...              True   \n",
                            "2   How many tourists visited New York City in 2019?             False   \n",
                            "3            What are the airports in New York City?              True   \n",
                            "4  When was the first documented visit into New Y...              True   \n",
                            "\n",
                            "   Correct Source  \n",
                            "0            True  \n",
                            "1            True  \n",
                            "2            True  \n",
                            "3            True  \n",
                            "4            True  "
                        ]
                    },
                    "execution_count": 502,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "analyze_outcome(outcomes_list_gpt3)"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "c4d0b3eb",
            "metadata": {},
            "source": [
                "### List Index + ChatGPT"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 23,
            "id": "f146c74e",
            "metadata": {},
            "outputs": [],
            "source": [
                "outcomes_list_chatgpt = bm.test(list_index, llm_predictor_chatgpt, response_mode=\"tree_summarize\", use_async=True)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 24,
            "id": "8eb9d392",
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<div>\n",
                            "<style scoped>\n",
                            "    .dataframe tbody tr th:only-of-type {\n",
                            "        vertical-align: middle;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe tbody tr th {\n",
                            "        vertical-align: top;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe thead th {\n",
                            "        text-align: right;\n",
                            "    }\n",
                            "</style>\n",
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th></th>\n",
                            "      <th>Test Query</th>\n",
                            "      <th>Correct Response</th>\n",
                            "      <th>Correct Source</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <th>0</th>\n",
                            "      <td>What battles took place in New York City in th...</td>\n",
                            "      <td>False</td>\n",
                            "      <td>True</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>1</th>\n",
                            "      <td>Who was elected as the mayor after the Great D...</td>\n",
                            "      <td>False</td>\n",
                            "      <td>True</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>2</th>\n",
                            "      <td>How many tourists visited New York City in 2019?</td>\n",
                            "      <td>False</td>\n",
                            "      <td>True</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>3</th>\n",
                            "      <td>What are the airports in New York City?</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>4</th>\n",
                            "      <td>When was the first documented visit into New Y...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>\n",
                            "</div>"
                        ],
                        "text/plain": [
                            "                                          Test Query  Correct Response  \\\n",
                            "0  What battles took place in New York City in th...             False   \n",
                            "1  Who was elected as the mayor after the Great D...             False   \n",
                            "2   How many tourists visited New York City in 2019?             False   \n",
                            "3            What are the airports in New York City?              True   \n",
                            "4  When was the first documented visit into New Y...              True   \n",
                            "\n",
                            "   Correct Source  \n",
                            "0            True  \n",
                            "1            True  \n",
                            "2            True  \n",
                            "3            True  \n",
                            "4            True  "
                        ]
                    },
                    "execution_count": 24,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "analyze_outcome(outcomes_list_chatgpt)"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "38fc1438",
            "metadata": {},
            "source": [
                "### Vector Store Index + GPT4 "
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 487,
            "id": "5349d1e7",
            "metadata": {},
            "outputs": [],
            "source": [
                "outcomes_vector_gpt4 = bm.test(vector_index, llm_predictor_gpt4)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 488,
            "id": "7fc53e19",
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<div>\n",
                            "<style scoped>\n",
                            "    .dataframe tbody tr th:only-of-type {\n",
                            "        vertical-align: middle;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe tbody tr th {\n",
                            "        vertical-align: top;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe thead th {\n",
                            "        text-align: right;\n",
                            "    }\n",
                            "</style>\n",
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th></th>\n",
                            "      <th>Test Query</th>\n",
                            "      <th>Correct Response</th>\n",
                            "      <th>Correct Source</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <th>0</th>\n",
                            "      <td>What battles took place in New York City in th...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>1</th>\n",
                            "      <td>Who was elected as the mayor during the Great ...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>2</th>\n",
                            "      <td>How many tourists visited New York City in 2019?</td>\n",
                            "      <td>False</td>\n",
                            "      <td>False</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>3</th>\n",
                            "      <td>What are the airports in New York City?</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>4</th>\n",
                            "      <td>When was the first documented visit into New Y...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>\n",
                            "</div>"
                        ],
                        "text/plain": [
                            "                                          Test Query  Correct Response  \\\n",
                            "0  What battles took place in New York City in th...              True   \n",
                            "1  Who was elected as the mayor during the Great ...              True   \n",
                            "2   How many tourists visited New York City in 2019?             False   \n",
                            "3            What are the airports in New York City?              True   \n",
                            "4  When was the first documented visit into New Y...              True   \n",
                            "\n",
                            "   Correct Source  \n",
                            "0            True  \n",
                            "1            True  \n",
                            "2           False  \n",
                            "3            True  \n",
                            "4            True  "
                        ]
                    },
                    "execution_count": 488,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "analyze_outcome(outcomes_vector_gpt4)"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "70eb711f",
            "metadata": {},
            "source": [
                "### Vector Store Index + GPT3"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 644,
            "id": "e35ebdf9",
            "metadata": {},
            "outputs": [],
            "source": [
                "outcomes_vector_gpt3 = bm.test(vector_index, llm_predictor_gpt3)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 645,
            "id": "95c49697",
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<div>\n",
                            "<style scoped>\n",
                            "    .dataframe tbody tr th:only-of-type {\n",
                            "        vertical-align: middle;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe tbody tr th {\n",
                            "        vertical-align: top;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe thead th {\n",
                            "        text-align: right;\n",
                            "    }\n",
                            "</style>\n",
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th></th>\n",
                            "      <th>Test Query</th>\n",
                            "      <th>Correct Response</th>\n",
                            "      <th>Correct Source</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <th>0</th>\n",
                            "      <td>What battles took place in New York City in th...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>1</th>\n",
                            "      <td>Who was elected as the mayor after the Great D...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>False</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>2</th>\n",
                            "      <td>How many tourists visited New York City in 2019?</td>\n",
                            "      <td>False</td>\n",
                            "      <td>False</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>3</th>\n",
                            "      <td>What are the airports in New York City?</td>\n",
                            "      <td>True</td>\n",
                            "      <td>False</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>4</th>\n",
                            "      <td>When was the first documented visit into New Y...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>False</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>\n",
                            "</div>"
                        ],
                        "text/plain": [
                            "                                          Test Query  Correct Response  \\\n",
                            "0  What battles took place in New York City in th...              True   \n",
                            "1  Who was elected as the mayor after the Great D...              True   \n",
                            "2   How many tourists visited New York City in 2019?             False   \n",
                            "3            What are the airports in New York City?              True   \n",
                            "4  When was the first documented visit into New Y...              True   \n",
                            "\n",
                            "   Correct Source  \n",
                            "0            True  \n",
                            "1           False  \n",
                            "2           False  \n",
                            "3           False  \n",
                            "4           False  "
                        ]
                    },
                    "execution_count": 645,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "analyze_outcome(outcomes_vector_gpt3)"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "a36ba2ee",
            "metadata": {},
            "source": [
                "# LLM based Evaluation"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 646,
            "id": "59ff561c",
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<div>\n",
                            "<style scoped>\n",
                            "    .dataframe tbody tr th:only-of-type {\n",
                            "        vertical-align: middle;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe tbody tr th {\n",
                            "        vertical-align: top;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe thead th {\n",
                            "        text-align: right;\n",
                            "    }\n",
                            "</style>\n",
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th></th>\n",
                            "      <th>Test Query</th>\n",
                            "      <th>Correct Response</th>\n",
                            "      <th>Correct Source</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <th>0</th>\n",
                            "      <td>What battles took place in New York City in th...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>1</th>\n",
                            "      <td>Who was elected as the mayor after the Great D...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>False</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>2</th>\n",
                            "      <td>How many tourists visited New York City in 2019?</td>\n",
                            "      <td>False</td>\n",
                            "      <td>False</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>3</th>\n",
                            "      <td>What are the airports in New York City?</td>\n",
                            "      <td>True</td>\n",
                            "      <td>False</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>4</th>\n",
                            "      <td>When was the first documented visit into New Y...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>False</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>\n",
                            "</div>"
                        ],
                        "text/plain": [
                            "                                          Test Query  Correct Response  \\\n",
                            "0  What battles took place in New York City in th...              True   \n",
                            "1  Who was elected as the mayor after the Great D...              True   \n",
                            "2   How many tourists visited New York City in 2019?             False   \n",
                            "3            What are the airports in New York City?              True   \n",
                            "4  When was the first documented visit into New Y...              True   \n",
                            "\n",
                            "   Correct Source  \n",
                            "0            True  \n",
                            "1           False  \n",
                            "2           False  \n",
                            "3           False  \n",
                            "4           False  "
                        ]
                    },
                    "execution_count": 646,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "analyze_outcome(outcomes_vector_gpt3)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 647,
            "id": "e4ffaca6",
            "metadata": {},
            "outputs": [],
            "source": [
                "eval_gpt4 = analyze_outcome_llm(outcomes_vector_gpt3, llm_predictor_gpt4)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 657,
            "id": "85c4e415",
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<div>\n",
                            "<style scoped>\n",
                            "    .dataframe tbody tr th:only-of-type {\n",
                            "        vertical-align: middle;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe tbody tr th {\n",
                            "        vertical-align: top;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe thead th {\n",
                            "        text-align: right;\n",
                            "    }\n",
                            "</style>\n",
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th></th>\n",
                            "      <th>Test Query</th>\n",
                            "      <th>Correct Response (LLM)</th>\n",
                            "      <th>Correct Source (LLM)</th>\n",
                            "      <th>Eval (LLM)</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <th>0</th>\n",
                            "      <td>What battles took place in New York City in th...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "      <td>Context is relevant: True\\nAnswer is correct: ...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>1</th>\n",
                            "      <td>Who was elected as the mayor after the Great D...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>False</td>\n",
                            "      <td>Context is relevant: False\\nAnswer is correct:...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>2</th>\n",
                            "      <td>How many tourists visited New York City in 2019?</td>\n",
                            "      <td>True</td>\n",
                            "      <td>False</td>\n",
                            "      <td>Context is relevant: False\\nAnswer is correct:...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>3</th>\n",
                            "      <td>What are the airports in New York City?</td>\n",
                            "      <td>True</td>\n",
                            "      <td>False</td>\n",
                            "      <td>Context is relevant: False\\nAnswer is correct:...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>4</th>\n",
                            "      <td>When was the first documented visit into New Y...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>False</td>\n",
                            "      <td>Context is relevant: False\\nAnswer is correct:...</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>\n",
                            "</div>"
                        ],
                        "text/plain": [
                            "                                          Test Query  Correct Response (LLM)  \\\n",
                            "0  What battles took place in New York City in th...                    True   \n",
                            "1  Who was elected as the mayor after the Great D...                    True   \n",
                            "2   How many tourists visited New York City in 2019?                    True   \n",
                            "3            What are the airports in New York City?                    True   \n",
                            "4  When was the first documented visit into New Y...                    True   \n",
                            "\n",
                            "   Correct Source (LLM)                                         Eval (LLM)  \n",
                            "0                  True  Context is relevant: True\\nAnswer is correct: ...  \n",
                            "1                 False  Context is relevant: False\\nAnswer is correct:...  \n",
                            "2                 False  Context is relevant: False\\nAnswer is correct:...  \n",
                            "3                 False  Context is relevant: False\\nAnswer is correct:...  \n",
                            "4                 False  Context is relevant: False\\nAnswer is correct:...  "
                        ]
                    },
                    "execution_count": 657,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "eval_gpt4"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 651,
            "id": "3efb66d6",
            "metadata": {},
            "outputs": [],
            "source": [
                "eval_chatgpt = analyze_outcome_llm(outcomes_vector_gpt3, llm_predictor_chatgpt)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 652,
            "id": "4c452767",
            "metadata": {
                "scrolled": true
            },
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<div>\n",
                            "<style scoped>\n",
                            "    .dataframe tbody tr th:only-of-type {\n",
                            "        vertical-align: middle;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe tbody tr th {\n",
                            "        vertical-align: top;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe thead th {\n",
                            "        text-align: right;\n",
                            "    }\n",
                            "</style>\n",
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th></th>\n",
                            "      <th>Test Query</th>\n",
                            "      <th>Correct Response (LLM)</th>\n",
                            "      <th>Correct Source (LLM)</th>\n",
                            "      <th>Eval (LLM)</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <th>0</th>\n",
                            "      <td>What battles took place in New York City in th...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "      <td>\\n\\nContext is relevant: True\\nAnswer is corre...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>1</th>\n",
                            "      <td>Who was elected as the mayor after the Great D...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "      <td>\\n\\nContext is relevant: True\\nAnswer is corre...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>2</th>\n",
                            "      <td>How many tourists visited New York City in 2019?</td>\n",
                            "      <td>False</td>\n",
                            "      <td>False</td>\n",
                            "      <td>\\n\\nContext is relevant: False\\nAnswer is corr...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>3</th>\n",
                            "      <td>What are the airports in New York City?</td>\n",
                            "      <td>True</td>\n",
                            "      <td>False</td>\n",
                            "      <td>\\n\\nContext is relevant: False\\nAnswer is corr...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>4</th>\n",
                            "      <td>When was the first documented visit into New Y...</td>\n",
                            "      <td>False</td>\n",
                            "      <td>True</td>\n",
                            "      <td>\\n\\nContext is relevant: True\\nAnswer is corre...</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>\n",
                            "</div>"
                        ],
                        "text/plain": [
                            "                                          Test Query  Correct Response (LLM)  \\\n",
                            "0  What battles took place in New York City in th...                    True   \n",
                            "1  Who was elected as the mayor after the Great D...                    True   \n",
                            "2   How many tourists visited New York City in 2019?                   False   \n",
                            "3            What are the airports in New York City?                    True   \n",
                            "4  When was the first documented visit into New Y...                   False   \n",
                            "\n",
                            "   Correct Source (LLM)                                         Eval (LLM)  \n",
                            "0                  True  \\n\\nContext is relevant: True\\nAnswer is corre...  \n",
                            "1                  True  \\n\\nContext is relevant: True\\nAnswer is corre...  \n",
                            "2                 False  \\n\\nContext is relevant: False\\nAnswer is corr...  \n",
                            "3                 False  \\n\\nContext is relevant: False\\nAnswer is corr...  \n",
                            "4                  True  \\n\\nContext is relevant: True\\nAnswer is corre...  "
                        ]
                    },
                    "execution_count": 652,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "eval_chatgpt"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 649,
            "id": "61e8dad2",
            "metadata": {},
            "outputs": [],
            "source": [
                "eval_gpt3 = analyze_outcome_llm(outcomes_vector_gpt3, llm_predictor_gpt3)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 650,
            "id": "170400c3",
            "metadata": {
                "scrolled": true
            },
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<div>\n",
                            "<style scoped>\n",
                            "    .dataframe tbody tr th:only-of-type {\n",
                            "        vertical-align: middle;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe tbody tr th {\n",
                            "        vertical-align: top;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe thead th {\n",
                            "        text-align: right;\n",
                            "    }\n",
                            "</style>\n",
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th></th>\n",
                            "      <th>Test Query</th>\n",
                            "      <th>Correct Response (LLM)</th>\n",
                            "      <th>Correct Source (LLM)</th>\n",
                            "      <th>Eval (LLM)</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <th>0</th>\n",
                            "      <td>What battles took place in New York City in th...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "      <td>\\n\\nContext is relevant: True\\nAnswer is corre...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>1</th>\n",
                            "      <td>Who was elected as the mayor after the Great D...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "      <td>\\n\\nContext is relevant: True\\nAnswer is corre...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>2</th>\n",
                            "      <td>How many tourists visited New York City in 2019?</td>\n",
                            "      <td>False</td>\n",
                            "      <td>False</td>\n",
                            "      <td>\\n\\nContext is relevant: False\\nAnswer is corr...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>3</th>\n",
                            "      <td>What are the airports in New York City?</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "      <td>\\n\\nContext is relevant: True\\nAnswer is corre...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>4</th>\n",
                            "      <td>When was the first documented visit into New Y...</td>\n",
                            "      <td>True</td>\n",
                            "      <td>True</td>\n",
                            "      <td>\\n\\nContext is relevant: True\\nAnswer is corre...</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>\n",
                            "</div>"
                        ],
                        "text/plain": [
                            "                                          Test Query  Correct Response (LLM)  \\\n",
                            "0  What battles took place in New York City in th...                    True   \n",
                            "1  Who was elected as the mayor after the Great D...                    True   \n",
                            "2   How many tourists visited New York City in 2019?                   False   \n",
                            "3            What are the airports in New York City?                    True   \n",
                            "4  When was the first documented visit into New Y...                    True   \n",
                            "\n",
                            "   Correct Source (LLM)                                         Eval (LLM)  \n",
                            "0                  True  \\n\\nContext is relevant: True\\nAnswer is corre...  \n",
                            "1                  True  \\n\\nContext is relevant: True\\nAnswer is corre...  \n",
                            "2                 False  \\n\\nContext is relevant: False\\nAnswer is corr...  \n",
                            "3                  True  \\n\\nContext is relevant: True\\nAnswer is corre...  \n",
                            "4                  True  \\n\\nContext is relevant: True\\nAnswer is corre...  "
                        ]
                    },
                    "execution_count": 650,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "eval_gpt3"
            ]
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": "Python 3 (ipykernel)",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.10.9"
        }
    },
    "nbformat": 4,
    "nbformat_minor": 5
}
